# -*- coding: utf-8 -*-
"""
Created on Thu Oct 19 09:28:38 2023

@author: xtp

"""

import numpy as np
from hyperimage import HyperImage
from smalltools import Files, Table, Tools
from dataprocess import DataLoad, PreProcessing, FeatureSelection, QuantitativeAnalysis
from multiprocessing import cpu_count
import concurrent.futures
from tqdm import tqdm
import pandas as pd
import numpy as np
import warnings
import os
import multiprocessing
from multiprocessing.dummy import Pool as ThreadPool
from functools import partial
import warnings
from sklearn.cross_decomposition import PLSRegression
from sklearn.preprocessing import scale,MinMaxScaler,Normalizer,StandardScaler
from sklearn.metrics import mean_squared_error,r2_score,mean_absolute_error
from sklearn.neural_network import MLPRegressor
import numpy as np
import matplotlib.pyplot as plt



def Outlier_Dataload(dataset, path=''):
    
    name = dataset[0]
    df = dataset[1]
    folder = path + name + '/Pre-processing/'
    Files.mkdirPath(folder)
    
    df_train, df_test = DataLoad.outlier_dataload(df, check_threshold=3.5,
                    check_band_num=5, split_method='ks', path=folder+'Outlier',
                    task='regression')
    
    return name, (df_train, df_test)

def Preprocess_Feature(data, methods, path):

    name, (df_train, df_test) = data
    label, method = methods
    folder = path + name + '/Pre-processing/'

    # PreProcess
    df_train_copy = df_train.copy(deep=True)
    df_test_copy = df_test.copy(deep=True)
    preprocess_train = PreProcessing(df_train_copy)
    preprocess_test = PreProcessing(df_test_copy)
    processed_train = preprocess_train.PreProcess(methods=method, 
                                                  path=folder+'Train_')
    processed_test = preprocess_test.PreProcess(methods=method, 
                                                path=folder+'Test_')

    return [[(name, label, 'Full'), (processed_train, processed_test)]]


def PLSR_GridSearch(result, n_features, path):
    
    label = result[0]
    folder = path + label[0] + '/Modeling/' + label[1] + '-' + label[2] + '/'
    Files.mkdirPath(folder)
    
    df_train, df_test = result[1]
    df_train.replace([np.inf,-np.inf,np.nan], 0)
    df_test.replace([np.inf,-np.inf,np.nan], 0)
    df_train = df_train.iloc[:, :n_features]
    df_test = df_test.iloc[:, :n_features]
    qa = QuantitativeAnalysis(df_train, df_test, method='PLSR')
    estimator = qa.getEstimator()
    params = qa.getParams()
    result_feature_G = qa.PLSR_cv_test(label)
    result_feature_G.to_csv(folder + 'PLSR_result.csv', index=False)
    
    return result_feature_G

def PLSR_Plot(result, wavelengths, X_max = 0, path=''):
    
    label1 = result.iloc[0,0][0]
    label2 = result.iloc[0,0][1] + '-' + result.iloc[0,0][2]
    folder = path + label1 + '/Modeling/' + label2 + '/'
    
    if X_max == 0:
        X_max = result.shape[0]

    x = list(range(1, X_max+1))
    fig = plt.figure(figsize=(6,5))
    
    i = X_max-1
    plt.plot(x, result.loc[:i, 'R2_Train'], 'o-',label="R2_Train", color="r")
    plt.plot(x, result.loc[:i, 'R2_Test'], 'o-',label="R2_Test", color="g")
    plt.plot(x, result.loc[:i, 'R2_CV'], 'o-',label="R2_CV",color="b")
    plt.xlabel("N components")
    plt.ylabel("R-Square")
    plt.legend(loc="best")
    plt.rc('font',family='Times New Roman')
    plt.rcParams['font.size'] = 10
    plt.title('PLSR GridSearch R2', fontsize=20)
    if path == '':
        plt.show()
    else:
        plt.savefig(folder+'PLSR_R2.png', dpi=300)
    plt.close()
    
    plt.plot(x, result.loc[:i, 'RMSE_Train'], 'o-',label="RMSE_Train", color="r")
    plt.plot(x, result.loc[:i, 'RMSE_Test'], 'o-',label="RMSE_Test", color="g")
    plt.plot(x, result.loc[:i, 'RMSE_CV'], 'o-',label="RMSE_CV",color="b")
    plt.xlabel("N components")
    plt.ylabel("Root Mean Squard Error")
    plt.legend(loc="best")
    plt.rc('font',family='Times New Roman')
    plt.rcParams['font.size'] = 10
    plt.title('PLSR GridSearch RMSE', fontsize=20)
    if path == '':
        plt.show()
    else:
        plt.savefig(folder+'PLSR_RMSE.png', dpi=300)
    plt.close()
    
    for j in list(range(X_max)):
        vip = result.loc[j, 'VIPs']
        plt.plot(wavelengths, vip, 'o-', label="component_"+str(j+1))
        plt.xlabel("wavelengths")
        plt.ylabel("Variable Importance in Projection")
        plt.legend(loc="best")
        plt.rc('font',family='Times New Roman')
        plt.rcParams['font.size'] = 10
        plt.title('PLSR VIP', fontsize=20)
    if path == '':
        plt.show()
    else:
        plt.savefig(folder+'PLSR_VIP.png', dpi=300)
    plt.close()
    

if __name__ == '__main__':
    
    warnings.filterwarnings("ignore")
    
    cores = cpu_count()
    
    path_processing = 'D:/Data/2023-Herbicide/Processing/'
    ext_processing = ['*.hdr']
    path_result = 'D:/Data/2023-Herbicide/Result/'
    ext_result = ['*.csv']
    Files.mkdirPath(path_result)
    
    #%%
    """ OutlierCheck and DataLoad """
    
    dataset = []
    process = Files(path_processing, ext_result, flag='all')
    for path, name in zip(process.filesWithPath, process.filesNoExt):
        table_processing = Table.read(path, header=0, index_col=[0,1,2,3,4,5,6,7], sep=',')
        df = table_processing.DataFrame.sort_index()
        df = df.reset_index(level=[0,1,2,3,4,5,6], drop=True)
        dataset.append([name, df])
    path = [path_result]*len(dataset)
    with concurrent.futures.ProcessPoolExecutor(max_workers=cores) as executor:
        results1 = list(tqdm(executor.map(Outlier_Dataload, dataset, path), 
                        position=0, ncols=80, leave=True, total=len(dataset), 
                        desc='OuterCheck & DataLoad', colour='blue'))
    results_list1 = [result for result in results1]
    
    #%%
    ''' Preprocessing and FeatureEngineering '''
    
    preprocess = [['D1', ['SG','MSC','D1','SS']], ['D2', ['SG','MSC','D2','SS']]]
    pre = [method for method in preprocess]*len(results_list1)
    res = [df for df in results_list1 for i in range(len(preprocess))]
    path = [path_result]*len(res)
    with concurrent.futures.ProcessPoolExecutor(max_workers=cores) as executor:
        results2 = list(tqdm(executor.map(Preprocess_Feature,res,pre,path), 
                        position=0, ncols=80, leave=True, total=len(res), 
                        desc='Preprocess & Features', colour='red'))
    results_list2 = []
    [results_list2.extend(result) for result in results2]
    
    #%%
    ''' Modeling and Evaluation '''
    
    results_list3 = []
    
    for result2 in tqdm(results_list2, position=0, ncols=80, leave=False, 
        total=len(results_list2), desc='PLSR_GridSearch', colour='magenta'):
        
        results_list3_1 = []
        results_list3_2 = []
        wavelengths = result2[1][1].columns
        
        if result2[0][2] != 'Full':
            
            diff_list = Tools.diff_list(len(result2[1][0].columns), step=5)
            result_1 = [result2]*len(diff_list)
            path = [path_result]*len(diff_list)
            with concurrent.futures.ProcessPoolExecutor(max_workers=cores) as executor:
                results3_1 = executor.map(PLSR_GridSearch, result_1, diff_list, path)
            for res in results3_1:
                PLSR_Plot(res, wavelengths, X_max=10, path=path_result)
                if len(res) != 0:
                    results_list3_1.append(res)
            results_list3.extend(results_list3_1)
            
        else:
            
            diff_list = [len(result2[1][0].columns)]
            result_1 = [result2]
            path = [path_result]
            with concurrent.futures.ProcessPoolExecutor(max_workers=cores) as executor:
                results3_2 = executor.map(PLSR_GridSearch, result_1, diff_list, path)
            for res in results3_2:
                PLSR_Plot(res, wavelengths, X_max=10, path=path_result)
                if len(res) != 0:
                    results_list3_2.append(res)
            results_list3.extend(results_list3_2)

    Result = pd.concat(results_list3, axis=0)
    Result.to_csv(path_result + 'Result_Summary.csv', index=False)
        
        
        
       










